{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install statsmodels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorboard.backend.event_processing.event_accumulator import EventAccumulator\n",
    "import glob\n",
    "import os\n",
    "import fnmatch\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy import interpolate\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.stats as st\n",
    "import statsmodels.stats.api as sms\n",
    "import colorsys\n",
    "from matplotlib.ticker import (MultipleLocator, FormatStrFormatter)\n",
    "\n",
    "\n",
    "\n",
    "## Baseline based on the normal noise model\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180225_035402_mini_imagenet_repeat_num_self_attention_splits_task_encoder_self_attention/'\n",
    "experiement_patterns = ['*num_self_attention_splits=1*', '*num_self_attention_splits=2*', \n",
    "                        '*num_self_attention_splits=4*']\n",
    "\n",
    "# ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180225_122144_mini_imagenet_repeat_num_self_attention_splits_fc_dropout_task_encoder_self_attention/'\n",
    "# experiement_patterns = ['*fc_dropout=0.2;num_self_attention_splits=4*',\n",
    "#                        '*fc_dropout=None;num_self_attention_splits=4*',\n",
    "#                        '*fc_dropout=0.5;num_self_attention_splits=4*',\n",
    "#                        '*fc_dropout=0.2;num_self_attention_splits=1*',\n",
    "#                        '*fc_dropout=None;num_self_attention_splits=1*',\n",
    "#                        '*fc_dropout=0.5;num_self_attention_splits=1*']\n",
    "# experiement_patterns = ['*fc_dropout=None*num_self_attention_splits=4*',\n",
    "#                        '*fc_dropout=None*num_self_attention_splits=1*']\n",
    "\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180227_034716_mini_imagenet_task_encoder_repeat_attention_num_filters_embedding_pooled_task_encoder_self_attention_patch/'\n",
    "experiement_patterns = ['*task_encoder=label_embed*attention_num_filters=64;embedding_pooled=True*',\n",
    "                        '*task_encoder=self_attention*attention_num_filters=64;embedding_pooled=True*',\n",
    "                        '*task_encoder=label_embed*attention_num_filters=32;embedding_pooled=True*',\n",
    "                        '*task_encoder=self_attention*attention_num_filters=32;embedding_pooled=True*',\n",
    "                        '*task_encoder=label_embed*attention_num_filters=16;embedding_pooled=True*',\n",
    "                        '*task_encoder=self_attention*attention_num_filters=16;embedding_pooled=True*',]\n",
    "\n",
    "# experiement_patterns = ['*attention_num_filters=64;embedding_pooled=True*',\n",
    "#                         '*attention_num_filters=32;embedding_pooled=True*',\n",
    "#                         '*attention_num_filters=16;embedding_pooled=True*',]\n",
    "\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180227_213958_mini_imagenet_repeat_attention_num_filters_class_embed_size_task_encoder_new_hyperparams_test/'\n",
    "experiement_patterns = ['*attention_num_filters=64*class_embed_size=16*',\n",
    "                        '*attention_num_filters=64*class_embed_size=32*',\n",
    "                        '*attention_num_filters=64*class_embed_size=64*',\n",
    "                        '*attention_num_filters=128*class_embed_size=16*',\n",
    "                        '*attention_num_filters=128*class_embed_size=32*',\n",
    "                        '*attention_num_filters=128*class_embed_size=64*',]\n",
    "\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180228_121015_mini_imagenet_repeat_attention_num_filters_class_embed_size_task_encoder_new_hyperparams_test/'\n",
    "experiement_patterns = ['*attention_num_filters=64*class_embed_size=16*',\n",
    "                        '*attention_num_filters=64*class_embed_size=32*',\n",
    "                        '*attention_num_filters=64*class_embed_size=64*',\n",
    "                        '*attention_num_filters=32*class_embed_size=16*',\n",
    "                        '*attention_num_filters=32*class_embed_size=32*',\n",
    "                        '*attention_num_filters=32*class_embed_size=64*',]\n",
    "\n",
    "\n",
    "# ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180301_011738_mini_imagenet_repeat_number_of_steps_task_encoder_num_steps/'\n",
    "# experiement_patterns = ['*number_of_steps=104000*']\n",
    "\n",
    "\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180301_112415_mini_imagenet_repeat_weights_initializer_factor_task_encoder_num_steps/'\n",
    "experiement_patterns = ['*weights_initializer_factor=1*',\n",
    "                        '*weights_initializer_factor=0.5*',\n",
    "                        '*weights_initializer_factor=0.2*',]\n",
    "\n",
    "\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180301_112900_mini_imagenet_num_attention_models_repeat_task_encoder_num_attention_models/'\n",
    "experiement_patterns = ['*num_attention_models=2*',\n",
    "                        '*num_attention_models=4*',\n",
    "                        '*num_attention_models=6*',]\n",
    "\n",
    "# 6 attention models\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180302_015239_mini_imagenet_repeat_weights_initializer_factor_task_encoder_6attention_models_noise_weight/'\n",
    "experiement_patterns = ['*weights_initializer_factor=0.5*',\n",
    "                        '*weights_initializer_factor=0.2*',\n",
    "                        '*weights_initializer_factor=0.1*',]\n",
    "\n",
    "# 6 attention models\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180302_113000_mini_imagenet_repeat_weights_initializer_factor_task_encoder_6attention_models_noise_weight/'\n",
    "experiement_patterns = ['*weights_initializer_factor=0.5*',\n",
    "                        '*weights_initializer_factor=0.2*',\n",
    "                        '*weights_initializer_factor=0.1*',]\n",
    "\n",
    "# Test highway and new implementation of sum \n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180302_194337_mini_imagenet_repeat_attention_fusion_task_encoder_testing_highway/'\n",
    "experiement_patterns = ['*attention_fusion=sum*','*attention_fusion=highway*',]\n",
    "\n",
    "# Test batch size 16\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180304_170453_mini_imagenet_repeat_number_of_steps_task_encoder_testing_highway/'\n",
    "experiement_patterns = ['*number_of_steps=156000*','*number_of_steps=208000*',]\n",
    "\n",
    "# FC dropout experiement\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180305_011527_mini_imagenet_repeat_number_of_steps_fc_dropout_task_encoder_fc_dropout/'\n",
    "experiement_patterns = ['*number_of_steps=156000*fc_dropout=None*',\n",
    "                       '*number_of_steps=156000*fc_dropout=0.1*',\n",
    "                       '*number_of_steps=156000*fc_dropout=0.2*',\n",
    "                       '*number_of_steps=156000*fc_dropout=0.5*',\n",
    "                       '*number_of_steps=208000*fc_dropout=None*',\n",
    "                       '*number_of_steps=208000*fc_dropout=0.1*',\n",
    "                       '*number_of_steps=208000*fc_dropout=0.2*',\n",
    "                       '*number_of_steps=208000*fc_dropout=0.5*',\n",
    "                       ]\n",
    "\n",
    "# Cosine learning rate annealing experiment\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180305_035219_mini_imagenet_repeat_task_encoder_cos_anneal/'\n",
    "experiement_patterns = ['*repeat*', ]\n",
    "\n",
    "# Experiment with more than one task per batch. This is usng piecewise constant anneal schedule 1/2, 3/4, 7/8\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180305_183801_mini_imagenet_repeat_num_tasks_per_batch_train_batch_size_task_encoder_cos_anneal'\n",
    "experiement_patterns = ['*train_batch_size=32*num_tasks_per_batch=1*',\n",
    "                       '*train_batch_size=32*num_tasks_per_batch=2*',\n",
    "                       '*train_batch_size=32*num_tasks_per_batch=3*',\n",
    "                       '*train_batch_size=16*num_tasks_per_batch=2*',\n",
    "                       '*train_batch_size=16*num_tasks_per_batch=3*',]\n",
    "\n",
    "# # Experiment with more than one task per batch. This is usng exponential equidistant learning rate decay with 3 segments\n",
    "# ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180305_211049_mini_imagenet_repeat_num_tasks_per_batch_train_batch_size_task_encoder_cos_anneal/'\n",
    "# experiement_patterns = ['*train_batch_size=32*num_tasks_per_batch=1*', \n",
    "#                         '*train_batch_size=16*num_tasks_per_batch=2*', \n",
    "#                         '*train_batch_size=32*num_tasks_per_batch=2*',\n",
    "#                         '*train_batch_size=8*num_tasks_per_batch=3*', \n",
    "#                         '*train_batch_size=16*num_tasks_per_batch=3*',\n",
    "#                         '*train_batch_size=32*num_tasks_per_batch=3*',\n",
    "#                        ]\n",
    "\n",
    "# # Baseline with 2 tasks per batch\n",
    "# ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180306_122149_mini_imagenet_repeat_number_of_steps_train_batch_size_lr_anneal_task_encoder_2tasks_batch/'\n",
    "# experiement_patterns = ['*train_batch_size=3*number_of_steps=1*lr_anneal=pwc*']\n",
    "\n",
    "# Experiment with 2, 3, 4 tasks per batch\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180306_184001_mini_imagenet_repeat_num_tasks_per_batch_task_encoder_234tasks_batch_pwc/'\n",
    "experiement_patterns = ['*num_tasks_per_batch=2*', '*num_tasks_per_batch=3*', '*num_tasks_per_batch=4*']\n",
    "\n",
    "\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180307_061837_mini_imagenet_number_of_steps_feat_extract_pretrain_repeat_task_encoder_pretrain_test/'\n",
    "experiement_patterns = ['*number_of_steps=5*feat_extract_pretrain=multitask*',\n",
    "                        '*number_of_steps=5*feat_extract_pretrain=finetune*',\n",
    "                        '*number_of_steps=5*feat_extract_pretrain=None*',\n",
    "                        '*number_of_steps=1*feat_extract_pretrain=multitask*',\n",
    "                        '*number_of_steps=1*feat_extract_pretrain=finetune*',\n",
    "                        '*number_of_steps=1*feat_extract_pretrain=None*'\n",
    "                       ]\n",
    "\n",
    "\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180307_184608_mini_imagenet_number_of_steps_feat_extract_pretrain_feat_extract_pretrain_lr_decay_n_repeat_task_encoder_pretrain_exploration/'\n",
    "experiement_patterns = ['*number_of_steps=5*feat_extract_pretrain_lr_decay_n=3*feat_extract_pretrain=multitask*',\n",
    "                        '*number_of_steps=5*feat_extract_pretrain_lr_decay_n=3*feat_extract_pretrain=finetune*',\n",
    "                        '*number_of_steps=5*feat_extract_pretrain_lr_decay_n=2*feat_extract_pretrain=multitask*',\n",
    "                        '*number_of_steps=5*feat_extract_pretrain_lr_decay_n=2*feat_extract_pretrain=finetune*',\n",
    "                        '*number_of_steps=2*feat_extract_pretrain_lr_decay_n=3*feat_extract_pretrain=multitask*',\n",
    "                        '*number_of_steps=2*feat_extract_pretrain_lr_decay_n=3*feat_extract_pretrain=finetune*',\n",
    "                        '*number_of_steps=2*feat_extract_pretrain_lr_decay_n=2*feat_extract_pretrain=multitask*',\n",
    "                        '*number_of_steps=2*feat_extract_pretrain_lr_decay_n=2*feat_extract_pretrain=finetune*',\n",
    "                       ]\n",
    "\n",
    "# Experiment with auxiliary task selection probability\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180308_163335_mini_imagenet_number_of_steps_feat_extract_pretrain_decay_rate_repeat_num_classes_pretrain_task_encoder_multitask_disjoint_datasets/'\n",
    "experiement_patterns = ['*number_of_steps=26000*feat_extract_pretrain_decay_rate=0.8*',\n",
    "                        '*number_of_steps=26000*feat_extract_pretrain_decay_rate=0.9;*',\n",
    "                        '*number_of_steps=26000*feat_extract_pretrain_decay_rate=0.99*',\n",
    "                        '*number_of_steps=52000*feat_extract_pretrain_decay_rate=0.8*',\n",
    "                        '*number_of_steps=52000*feat_extract_pretrain_decay_rate=0.9;*',\n",
    "                        '*number_of_steps=52000*feat_extract_pretrain_decay_rate=0.99*',]\n",
    "\n",
    "# New baseline, 8 attention splits\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180309_034509_mini_imagenet_num_attention_models_task_encoder_num_attention_layers_repeat_task_encoder_pretrain_test/'\n",
    "experiement_patterns = ['*num_attention_models=8*num_attention_layers=1*',\n",
    "                       '*num_attention_models=6*num_attention_layers=1*',\n",
    "                       '*num_attention_models=1*num_attention_layers=1*',]\n",
    "\n",
    "\n",
    "# New baseline, 8, 12, 16 attention splits and 32, 64, 128 filters\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180309_124026_mini_imagenet_num_attention_models_attention_num_filters_repeat_task_encoder_attention_filters_splits/'\n",
    "experiement_patterns = ['*num_attention_models=16*attention_num_filters=128*',\n",
    "                       '*num_attention_models=16*attention_num_filters=64*',\n",
    "                       '*num_attention_models=16*attention_num_filters=32*',]\n",
    "\n",
    "# Experiment, small number of steps\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180309_183505_mini_imagenet_number_of_steps_class_embed_size_repeat_task_encoder_class_embedding/'\n",
    "experiement_patterns = ['*number_of_steps=15600*class_embed_size=32*',\n",
    "                       '*number_of_steps=15600*class_embed_size=64*',]\n",
    "\n",
    "\n",
    "# # Experiment, not sharing the class embeddinggs\n",
    "# ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180309_205949_mini_imagenet_num_attention_layers_repeat_task_encoder_sharing_task_encoder_task_embedding_sharing/'\n",
    "# experiement_patterns = ['*num_attention_layers=1*task_encoder_sharing=None*',\n",
    "#                        '*num_attention_layers=1*task_encoder_sharing=layer*',\n",
    "#                        '*num_attention_layers=1*task_encoder_sharing=global*',]\n",
    "\n",
    "\n",
    "# Experiment, not sharing the class embeddinggs\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180310_151540_mini_imagenet_number_of_steps_class_embed_size_repeat_task_encoder_sharing_task_encoder_embedding_size_8_16_32/'\n",
    "experiement_patterns = ['*number_of_steps=26000*class_embed_size=16;task_encoder_sharing=None*',\n",
    "                       '*number_of_steps=26000*class_embed_size=16;task_encoder_sharing=layer*',\n",
    "                       '*number_of_steps=26000*class_embed_size=32;task_encoder_sharing=None*',\n",
    "                       '*number_of_steps=26000*class_embed_size=32;task_encoder_sharing=layer*',\n",
    "                       '*number_of_steps=15600*class_embed_size=16;task_encoder_sharing=None*',\n",
    "                       '*number_of_steps=15600*class_embed_size=16;task_encoder_sharing=layer*',\n",
    "                       '*number_of_steps=15600*class_embed_size=32;task_encoder_sharing=None*',\n",
    "                       '*number_of_steps=15600*class_embed_size=32;task_encoder_sharing=layer*',\n",
    "                       '*number_of_steps=26000*class_embed_size=8;task_encoder_sharing=None*',\n",
    "                       '*number_of_steps=26000*class_embed_size=8;task_encoder_sharing=layer*',\n",
    "                       '*number_of_steps=15600*class_embed_size=8;task_encoder_sharing=None*',\n",
    "                       '*number_of_steps=15600*class_embed_size=8;task_encoder_sharing=layer*',\n",
    "                       ]\n",
    "\n",
    "# Experiment, not sharing the class embeddinggs, more runs\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180311_041110_mini_imagenet_number_of_steps_class_embed_size_repeat_task_encoder_sharing_task_encoder_embedding_size_8_16_32/'\n",
    "experiement_patterns = ['*number_of_steps=26000*class_embed_size=8;task_encoder_sharing=None*',\n",
    "                       '*number_of_steps=26000*class_embed_size=8;task_encoder_sharing=layer*',\n",
    "                       '*number_of_steps=26000*class_embed_size=32;task_encoder_sharing=None*',\n",
    "                       '*number_of_steps=26000*class_embed_size=32;task_encoder_sharing=layer*',\n",
    "                       '*number_of_steps=15600*class_embed_size=8;task_encoder_sharing=None*',\n",
    "                       '*number_of_steps=15600*class_embed_size=8;task_encoder_sharing=layer*',\n",
    "                       '*number_of_steps=15600*class_embed_size=32;task_encoder_sharing=None*',\n",
    "                       '*number_of_steps=15600*class_embed_size=32;task_encoder_sharing=layer*',\n",
    "                       ]\n",
    "\n",
    "# Ablation study prototypical head, resnet, sgd, multitask\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180313_001359_mini_imagenet_encoder_classifier_link_feature_extractor_repeat_embedding_pooled_task_encoder_feature_extractor_ablation_more_steps/'\n",
    "experiement_patterns = ['*encoder_classifier_link=prototypical*feature_extractor=simple_conv_net;embedding_pooled=True*',\n",
    "                       '*encoder_classifier_link=prototypical*feature_extractor=simple_conv_net;embedding_pooled=False*',\n",
    "                        '*encoder_classifier_link=prototypical*feature_extractor=simple_res_net;embedding_pooled=True*',\n",
    "                       '*encoder_classifier_link=prototypical*feature_extractor=simple_res_net;embedding_pooled=False*',\n",
    "                       ]\n",
    "\n",
    "# Ablation study prototypical head and adam\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180313_112548_mini_imagenet_encoder_classifier_link_feature_extractor_repeat_embedding_pooled_task_encoder_prototypical_adam_no_multitask/'\n",
    "experiement_patterns = ['*encoder_classifier_link=prototypical*feature_extractor=simple_conv_net;embedding_pooled=True*',\n",
    "                       '*encoder_classifier_link=prototypical*feature_extractor=simple_conv_net;embedding_pooled=False*',\n",
    "                        '*encoder_classifier_link=prototypical*feature_extractor=simple_res_net;embedding_pooled=True*',\n",
    "                       '*encoder_classifier_link=prototypical*feature_extractor=simple_res_net;embedding_pooled=False*',\n",
    "                       ]\n",
    "\n",
    "# Ablation study prototypical head and adam, no L2 norm\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180313_164709_mini_imagenet_number_of_steps_feature_extractor_repeat_weight_decay_task_encoder_prototypical/'\n",
    "experiement_patterns = ['*number_of_steps=20000*weight_decay=0;feature_extractor=simple_conv_net*',\n",
    "                        '*number_of_steps=20000*weight_decay=0.0005;feature_extractor=simple_conv_net*',\n",
    "#                         '*number_of_steps=20000*weight_decay=0;feature_extractor=simple_res_net*',\n",
    "#                         '*number_of_steps=20000*weight_decay=0.0005;feature_extractor=simple_res_net*',\n",
    "                       ]\n",
    "\n",
    "# Replicating the prototypical nets results\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180314_014829_mini_imagenet_number_of_steps_repeat_train_batch_size_num_classes_train_task_encoder_prototypical/'\n",
    "experiement_patterns = ['*train_batch_size=300;number_of_steps=20000*num_classes_train=5*',\n",
    "                        '*train_batch_size=300;number_of_steps=20000*num_classes_train=20*',\n",
    "                        '*train_batch_size=300;number_of_steps=40000*num_classes_train=5*',\n",
    "                        '*train_batch_size=300;number_of_steps=40000*num_classes_train=20*',\n",
    "                        '*train_batch_size=75;number_of_steps=20000*num_classes_train=5*',\n",
    "                        '*train_batch_size=75;number_of_steps=20000*num_classes_train=20*',\n",
    "                        '*train_batch_size=75;number_of_steps=40000*num_classes_train=5*',\n",
    "                        '*train_batch_size=75;number_of_steps=40000*num_classes_train=20*',\n",
    "                       ]\n",
    "\n",
    "# Prototypical nets with res_net feature extractor\n",
    "# Replicating the prototypical nets results\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180314_021127_mini_imagenet_number_of_steps_feature_extractor_repeat_train_batch_size_num_classes_train_task_encoder_prototypical_20way_resnet/'\n",
    "experiement_patterns = ['*train_batch_size=300;number_of_steps=20000*feature_extractor=simple_res_net*num_classes_train=5*',\n",
    "#                         '*train_batch_size=300;number_of_steps=20000*feature_extractor=simple_res_net*num_classes_train=20*',\n",
    "                        '*train_batch_size=300;number_of_steps=40000*feature_extractor=simple_res_net*num_classes_train=5*',\n",
    "#                         '*train_batch_size=300;number_of_steps=40000*feature_extractor=simple_res_net*num_classes_train=20*',\n",
    "                        '*train_batch_size=75;number_of_steps=20000*feature_extractor=simple_res_net*num_classes_train=5*',\n",
    "                        '*train_batch_size=75;number_of_steps=20000*feature_extractor=simple_res_net*num_classes_train=20*',\n",
    "                        '*train_batch_size=75;number_of_steps=40000*feature_extractor=simple_res_net*num_classes_train=5*',\n",
    "                        '*train_batch_size=75;number_of_steps=40000*feature_extractor=simple_res_net*num_classes_train=20*',\n",
    "                       ]\n",
    "\n",
    "\n",
    "# Study of multitask training and adam for conv_net, \n",
    "# multitask does not help, probably with adam final learning rate for the aux task is not optimal\n",
    "# No significant difference, so class embedding is all we need\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180315_040436_mini_imagenet_feat_extract_pretrain_repeat_num_classes_train_task_encoder_prototypical_multitask/'\n",
    "experiement_patterns = [\n",
    "    '*num_classes_train=5*feat_extract_pretrain=multitask*',\n",
    "    '*num_classes_train=20*feat_extract_pretrain=multitask*',\n",
    "    '*num_classes_train=5*feat_extract_pretrain=None*',\n",
    "    '*num_classes_train=20*feat_extract_pretrain=None*',\n",
    "]\n",
    "\n",
    "# Experiemnt excluding the image features from the FC layer in attention model. Only use class embedding\n",
    "ROOT_DIR = '/mnt/home/boris/experiments_task_encoder/180314_233040_mini_imagenet_num_attention_models_attention_no_original_embedding_task_encoder_sharing_repeat_feat_extract_pretrain_decay_rate_num_classes_train_task_encoder_cut_original_embedding/'\n",
    "experiement_patterns = [\n",
    "    \"*num_attention_models=16*num_classes_train=5;task_encoder_sharing=None;attention_no_original_embedding=True;feat_extract_pretrain_decay_rate=0.9*\",\n",
    "    \"*num_attention_models=16*num_classes_train=5;task_encoder_sharing=None;attention_no_original_embedding=False;feat_extract_pretrain_decay_rate=0.9*\",\n",
    "]\n",
    "\n",
    "# Experiemnt multi-way training of resnet with adam and euclidian distance\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180314_191612_mini_imagenet_repeat_train_batch_size_num_classes_train_task_encoder_prototypical_multyway_resnet/'\n",
    "experiement_patterns = [\n",
    "    \"*train_batch_size=75*num_classes_train=5*\",\n",
    "    \"*train_batch_size=75*num_classes_train=10*\",\n",
    "    \"*train_batch_size=75*num_classes_train=20*\",\n",
    "    \"*train_batch_size=75*num_classes_train=30*\",\n",
    "]\n",
    "\n",
    "# Experiement using normalization by standard deviation in the prototypical context\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180315_153440_mini_imagenet_encoder_classifier_link_repeat_num_classes_train_task_encoder_prototypical_std_norm/'\n",
    "experiement_patterns = [\n",
    "    \"*encoder_classifier_link=prototypical*num_classes_train=20*\",\n",
    "    \"*encoder_classifier_link=prototypical*num_classes_train=5*\",\n",
    "    \"*encoder_classifier_link=std_normalized_euc_head*num_classes_train=20*\",\n",
    "    \"*encoder_classifier_link=std_normalized_euc_head*num_classes_train=5*\",\n",
    "]\n",
    "\n",
    "# Experiment with resnet, testing multitask and number of classes while training\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180315_160112_mini_imagenet_num_tasks_per_batch_feat_extract_pretrain_feature_extractor_repeat_num_classes_train_task_encoder_proto_resnet_multitask/'\n",
    "experiement_patterns = [\n",
    "    \"*num_classes_train=5*num_tasks_per_batch=2;feature_extractor=simple_res_net;feat_extract_pretrain=multitask*\",\n",
    "    \"*num_classes_train=20*num_tasks_per_batch=2;feature_extractor=simple_res_net;feat_extract_pretrain=multitask*\",\n",
    "    \"*num_classes_train=5*num_tasks_per_batch=2;feature_extractor=simple_res_net;feat_extract_pretrain=None*\",\n",
    "    \"*num_classes_train=20*num_tasks_per_batch=2;feature_extractor=simple_res_net;feat_extract_pretrain=None*\",\n",
    "    \n",
    "]\n",
    "\n",
    "# Experiemnt multiway training resnet with sgd\n",
    "'/mnt/home/boris/experiments_task_encoder/180315_205348_mini_imagenet_feat_extract_pretrain_feat_extract_pretrain_lr_decay_n_repeat_num_classes_train_task_encoder_proto_resnet_multitask'\n",
    "\n",
    "# Experiement Euclidian vs. Cosine distance with adam and non-pooled features\n",
    "'/mnt/home/boris/experiments_task_encoder/180316_135152_mini_imagenet_encoder_classifier_link_number_of_steps_repeat_num_classes_train_task_encoder_self_attention_euclidian/'\n",
    "\n",
    "# Experiemnt multiplying cosine distance by a scalar, alexnet and adam\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180317_175217_mini_imagenet_encoder_classifier_link_number_of_steps_repeat_cosine_multiplier_trainable_num_classes_train_cosine_multiplier_init_task_encoder_cosine_scaled_15/'\n",
    "experiement_patterns = [\n",
    "    \"*num_classes_train=20;encoder_classifier_link=prototypical*number_of_steps=80000*\",\n",
    "    \"*num_classes_train=20;encoder_classifier_link=cosine;cosine_multiplier_trainable=False;number_of_steps=80000;cosine_multiplier_init=15\",\n",
    "    \"*num_classes_train=5;encoder_classifier_link=prototypical*number_of_steps=80000*\",\n",
    "    \"*num_classes_train=5;encoder_classifier_link=cosine;cosine_multiplier_trainable=False;number_of_steps=80000;cosine_multiplier_init=15\",\n",
    "    \"*num_classes_train=20;encoder_classifier_link=prototypical*number_of_steps=40000*\",\n",
    "    \"*num_classes_train=20;encoder_classifier_link=cosine;cosine_multiplier_trainable=False;number_of_steps=40000;cosine_multiplier_init=15\",\n",
    "    \"*num_classes_train=5;encoder_classifier_link=prototypical*number_of_steps=40000*\",\n",
    "    \"*num_classes_train=5;encoder_classifier_link=cosine;cosine_multiplier_trainable=False;number_of_steps=40000;cosine_multiplier_init=15\",\n",
    "]\n",
    "\n",
    "# Experiemnt doing polynomial of euclidian distance, alexnet and adam\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180318_044135_mini_imagenet_encoder_classifier_link_number_of_steps_polynomial_metric_order_repeat_num_classes_train_task_encoder_polynomial_metric/'\n",
    "experiement_patterns = [\n",
    "    \"*encoder_classifier_link=polynomial;number_of_steps=40000*num_classes_train=5;polynomial_metric_order=1\",\n",
    "    \"*encoder_classifier_link=polynomial;number_of_steps=80000*num_classes_train=5;polynomial_metric_order=1\",\n",
    "    \"*encoder_classifier_link=polynomial;number_of_steps=40000*num_classes_train=5;polynomial_metric_order=5\",\n",
    "    \"*encoder_classifier_link=polynomial;number_of_steps=80000*num_classes_train=5;polynomial_metric_order=5\",\n",
    "]\n",
    "\n",
    "# Experiement estimating a polynomial of the Euclidian distance, resnet and SGD\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180318_054924_mini_imagenet_encoder_classifier_link_number_of_steps_polynomial_metric_order_repeat_num_classes_train_task_encoder_polynomial_resnet'\n",
    "experiement_patterns = [\n",
    "    \"*encoder_classifier_link=prototypical;number_of_steps=26000*num_classes_train=5*\",\n",
    "    \"*encoder_classifier_link=prototypical;number_of_steps=26000*num_classes_train=20*\",\n",
    "    \"*encoder_classifier_link=prototypical;number_of_steps=52000*num_classes_train=5*\",\n",
    "    \"*encoder_classifier_link=prototypical;number_of_steps=52000*num_classes_train=20*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=26000*num_classes_train=5;polynomial_metric_order=1*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=26000*num_classes_train=20;polynomial_metric_order=1*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=26000*num_classes_train=5;polynomial_metric_order=2*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=26000*num_classes_train=20;polynomial_metric_order=2*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=26000*num_classes_train=5;polynomial_metric_order=5*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=26000*num_classes_train=20;polynomial_metric_order=5*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=52000*num_classes_train=5;polynomial_metric_order=1*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=52000*num_classes_train=20;polynomial_metric_order=1*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=52000*num_classes_train=5;polynomial_metric_order=2*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=52000*num_classes_train=20;polynomial_metric_order=2*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=52000*num_classes_train=5;polynomial_metric_order=5*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=52000*num_classes_train=20;polynomial_metric_order=5*\",\n",
    "]\n",
    "\n",
    "# # Experiement estimating a polynomial of the Euclidian distance, resnet and SGD, varying polynomial degrees\n",
    "# ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180319_020058_mini_imagenet_encoder_classifier_link_number_of_steps_polynomial_metric_order_repeat_num_classes_train_task_encoder_polynomial_perceptron_resnet/'\n",
    "# experiement_patterns = [\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=26000*num_classes_train=5;polynomial_metric_order=1*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=26000*num_classes_train=5;polynomial_metric_order=2*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=26000*num_classes_train=5;polynomial_metric_order=3*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=26000*num_classes_train=5;polynomial_metric_order=4*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=26000*num_classes_train=5;polynomial_metric_order=5*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=26000*num_classes_train=20;polynomial_metric_order=1*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=26000*num_classes_train=20;polynomial_metric_order=2*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=26000*num_classes_train=20;polynomial_metric_order=3*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=26000*num_classes_train=20;polynomial_metric_order=4*\",\n",
    "#     \"*encoder_classifier_link=polynomial;number_of_steps=26000*num_classes_train=20;polynomial_metric_order=5*\",\n",
    "# ]\n",
    "\n",
    "# Experiement estimating a polynomial of the Euclidian distance, resnet and SGD, varying polynomial degrees, more repeats\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180319_173649_mini_imagenet_number_of_steps_polynomial_metric_order_repeat_num_classes_train_task_encoder_polynomial_perceptron_resnet/'\n",
    "experiement_patterns = [\n",
    "    \"*number_of_steps=25000*num_classes_train=5;polynomial_metric_order=1*\",\n",
    "    \"*number_of_steps=25000*num_classes_train=5;polynomial_metric_order=2*\",\n",
    "    \"*number_of_steps=25000*num_classes_train=5;polynomial_metric_order=3*\",\n",
    "    \"*number_of_steps=50000*num_classes_train=5;polynomial_metric_order=1*\",\n",
    "    \"*number_of_steps=50000*num_classes_train=5;polynomial_metric_order=2*\",\n",
    "    \"*number_of_steps=50000*num_classes_train=5;polynomial_metric_order=3*\",\n",
    "    \"*number_of_steps=25000*num_classes_train=10;polynomial_metric_order=1*\",\n",
    "    \"*number_of_steps=25000*num_classes_train=10;polynomial_metric_order=2*\",\n",
    "    \"*number_of_steps=25000*num_classes_train=10;polynomial_metric_order=3*\",\n",
    "    \"*number_of_steps=50000*num_classes_train=10;polynomial_metric_order=1*\",\n",
    "    \"*number_of_steps=50000*num_classes_train=10;polynomial_metric_order=2*\",\n",
    "    \"*number_of_steps=50000*num_classes_train=10;polynomial_metric_order=3*\",\n",
    "    \"*number_of_steps=25000*num_classes_train=20;polynomial_metric_order=1*\",\n",
    "    \"*number_of_steps=25000*num_classes_train=20;polynomial_metric_order=2*\",\n",
    "    \"*number_of_steps=25000*num_classes_train=20;polynomial_metric_order=3*\",\n",
    "    \"*number_of_steps=50000*num_classes_train=20;polynomial_metric_order=1*\",\n",
    "    \"*number_of_steps=50000*num_classes_train=20;polynomial_metric_order=2*\",\n",
    "    \"*number_of_steps=50000*num_classes_train=20;polynomial_metric_order=3*\",\n",
    "    \n",
    "]\n",
    "\n",
    "\n",
    "# Experiement estimating a polynomial of the Euclidian distance, resnet and SGD, varying polynomial degrees, \n",
    "# replicating results with tighter range of number_of_steps and num_classes_train, but less repeats\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180320_161741_mini_imagenet_number_of_steps_polynomial_metric_order_repeat_num_classes_train_task_encoder_polynomial_perceptron_resnet/'\n",
    "experiement_patterns = [\n",
    "    \"*number_of_steps=50000*num_classes_train=5;polynomial_metric_order=1*\",\n",
    "    \"*number_of_steps=50000*num_classes_train=5;polynomial_metric_order=2*\",\n",
    "    \"*number_of_steps=50000*num_classes_train=5;polynomial_metric_order=3*\",\n",
    "    \"*number_of_steps=50000*num_classes_train=10;polynomial_metric_order=1*\",\n",
    "    \"*number_of_steps=50000*num_classes_train=10;polynomial_metric_order=2*\",\n",
    "    \"*number_of_steps=50000*num_classes_train=10;polynomial_metric_order=3*\",\n",
    "    \"*number_of_steps=50000*num_classes_train=15;polynomial_metric_order=1*\",\n",
    "    \"*number_of_steps=50000*num_classes_train=15;polynomial_metric_order=2*\",\n",
    "    \"*number_of_steps=50000*num_classes_train=15;polynomial_metric_order=3*\",\n",
    "    \"*number_of_steps=40000*num_classes_train=5;polynomial_metric_order=1*\",\n",
    "    \"*number_of_steps=40000*num_classes_train=5;polynomial_metric_order=2*\",\n",
    "    \"*number_of_steps=40000*num_classes_train=5;polynomial_metric_order=3*\",\n",
    "    \"*number_of_steps=40000*num_classes_train=10;polynomial_metric_order=1*\",\n",
    "    \"*number_of_steps=40000*num_classes_train=10;polynomial_metric_order=2*\",\n",
    "    \"*number_of_steps=40000*num_classes_train=10;polynomial_metric_order=3*\",\n",
    "    \"*number_of_steps=40000*num_classes_train=15;polynomial_metric_order=1*\",\n",
    "    \"*number_of_steps=40000*num_classes_train=15;polynomial_metric_order=2*\",\n",
    "    \"*number_of_steps=40000*num_classes_train=15;polynomial_metric_order=3*\",\n",
    "    \"*number_of_steps=60000*num_classes_train=5;polynomial_metric_order=1*\",\n",
    "    \"*number_of_steps=60000*num_classes_train=5;polynomial_metric_order=2*\",\n",
    "    \"*number_of_steps=60000*num_classes_train=5;polynomial_metric_order=3*\",\n",
    "    \"*number_of_steps=60000*num_classes_train=10;polynomial_metric_order=1*\",\n",
    "    \"*number_of_steps=60000*num_classes_train=10;polynomial_metric_order=2*\",\n",
    "    \"*number_of_steps=60000*num_classes_train=10;polynomial_metric_order=3*\",\n",
    "    \"*number_of_steps=60000*num_classes_train=15;polynomial_metric_order=1*\",\n",
    "    \"*number_of_steps=60000*num_classes_train=15;polynomial_metric_order=2*\",\n",
    "    \"*number_of_steps=60000*num_classes_train=15;polynomial_metric_order=3*\",\n",
    "    \n",
    "]\n",
    "\n",
    "# Baseline with a polynomial of the Euclidian distance, resnet and SGD\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180321_150820_mini_imagenet_number_of_steps_polynomial_metric_order_repeat_num_classes_train_polynomial_baseline/'\n",
    "experiement_patterns = [\n",
    "    '*number_of_steps=50000*num_classes_train=10;polynomial_metric_order=1*',\n",
    "    '*number_of_steps=50000*num_classes_train=5;polynomial_metric_order=1*',\n",
    "    '*number_of_steps=50000*num_classes_train=10;polynomial_metric_order=2*',\n",
    "    '*number_of_steps=50000*num_classes_train=5;polynomial_metric_order=2*',\n",
    "    '*number_of_steps=50000*num_classes_train=10;polynomial_metric_order=3*',\n",
    "    '*number_of_steps=50000*num_classes_train=5;polynomial_metric_order=3*',\n",
    "]\n",
    "\n",
    "# Baseline with a polynomial of the Euclidian distance, resnet and SGD, less distance between learning rate annealing events\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180322_002424_mini_imagenet_number_of_steps_polynomial_metric_order_repeat_num_classes_train_polynomial_baseline/'\n",
    "experiement_patterns = [\n",
    "    '*number_of_steps=50000*num_classes_train=10;polynomial_metric_order=1*',\n",
    "    '*number_of_steps=50000*num_classes_train=5;polynomial_metric_order=1*',\n",
    "    '*number_of_steps=50000*num_classes_train=10;polynomial_metric_order=2*',\n",
    "    '*number_of_steps=50000*num_classes_train=5;polynomial_metric_order=2*',\n",
    "    '*number_of_steps=50000*num_classes_train=10;polynomial_metric_order=3*',\n",
    "    '*number_of_steps=50000*num_classes_train=5;polynomial_metric_order=3*',\n",
    "]\n",
    "\n",
    "# swish-1, selu, relu experiemnt\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180322_210010_mini_imagenet_number_of_steps_lr_anneal_repeat_activation_num_classes_train_swish-1-test/'\n",
    "experiement_patterns = [\n",
    "    '*activation=relu;lr_anneal=pwc;num_classes_train=5*',\n",
    "    '*activation=selu;lr_anneal=pwc;num_classes_train=5*',\n",
    "    '*activation=swish-1;lr_anneal=pwc;num_classes_train=5*',\n",
    "]\n",
    "\n",
    "# Baseline for the adam based optimizer with simple_conv_net, mini-imagenet, swish-1 activation\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180323_223118_mini_imagenet_encoder_classifier_link_number_of_steps_data_dir_metric_multiplier_init_metric_multiplier_trainable_repeat_num_classes_train_simple_conv_net_mini_imagenet_baseline/'\n",
    "experiement_patterns = [\n",
    "    '*metric_multiplier_trainable=False*num_classes_train=5;encoder_classifier_link=cosine;number_of_steps=40000;metric_multiplier_init=1',\n",
    "    '*metric_multiplier_trainable=True*num_classes_train=5;encoder_classifier_link=cosine;number_of_steps=40000;metric_multiplier_init=1',\n",
    "    '*metric_multiplier_trainable=False*num_classes_train=5;encoder_classifier_link=cosine;number_of_steps=40000;metric_multiplier_init=15',\n",
    "    '*metric_multiplier_trainable=True*num_classes_train=5;encoder_classifier_link=cosine;number_of_steps=40000;metric_multiplier_init=15',\n",
    "    '*metric_multiplier_trainable=False*num_classes_train=20;encoder_classifier_link=cosine;number_of_steps=40000;metric_multiplier_init=1',\n",
    "    '*metric_multiplier_trainable=True*num_classes_train=20;encoder_classifier_link=cosine;number_of_steps=40000;metric_multiplier_init=1',\n",
    "    '*metric_multiplier_trainable=False*num_classes_train=20;encoder_classifier_link=cosine;number_of_steps=40000;metric_multiplier_init=15',\n",
    "    '*metric_multiplier_trainable=True*num_classes_train=20;encoder_classifier_link=cosine;number_of_steps=40000;metric_multiplier_init=15',\n",
    "    '*num_classes_train=5;encoder_classifier_link=prototypical;number_of_steps=40000*',\n",
    "    '*num_classes_train=20;encoder_classifier_link=prototypical;number_of_steps=40000*',\n",
    "    '*num_classes_train=5;encoder_classifier_link=prototypical;number_of_steps=20000*',\n",
    "    '*num_classes_train=20;encoder_classifier_link=prototypical;number_of_steps=20000*',\n",
    "]\n",
    "\n",
    "# Baseline for the adam based optimizer with simple_conv_net, mini-imagenet, relu activation\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180326_012431_mini_imagenet_encoder_classifier_link_number_of_steps_data_dir_metric_multiplier_init_metric_multiplier_trainable_repeat_num_classes_train_simple_conv_net_mini_imagenet_baseline_with_relu/'\n",
    "experiement_patterns = [\n",
    "    '*metric_multiplier_trainable=False*num_classes_train=5;encoder_classifier_link=cosine;number_of_steps=40000;metric_multiplier_init=1',\n",
    "    '*metric_multiplier_trainable=True*num_classes_train=5;encoder_classifier_link=cosine;number_of_steps=40000;metric_multiplier_init=1',\n",
    "    '*metric_multiplier_trainable=False*num_classes_train=5;encoder_classifier_link=cosine;number_of_steps=40000;metric_multiplier_init=15',\n",
    "    '*metric_multiplier_trainable=True*num_classes_train=5;encoder_classifier_link=cosine;number_of_steps=40000;metric_multiplier_init=15',\n",
    "    '*metric_multiplier_trainable=False*num_classes_train=20;encoder_classifier_link=cosine;number_of_steps=40000;metric_multiplier_init=1',\n",
    "    '*metric_multiplier_trainable=True*num_classes_train=20;encoder_classifier_link=cosine;number_of_steps=40000;metric_multiplier_init=1',\n",
    "    '*metric_multiplier_trainable=False*num_classes_train=20;encoder_classifier_link=cosine;number_of_steps=40000;metric_multiplier_init=15',\n",
    "    '*metric_multiplier_trainable=True*num_classes_train=20;encoder_classifier_link=cosine;number_of_steps=40000;metric_multiplier_init=15',\n",
    "    '*num_classes_train=5;encoder_classifier_link=prototypical;number_of_steps=40000*',\n",
    "    '*num_classes_train=20;encoder_classifier_link=prototypical;number_of_steps=40000*',\n",
    "    '*num_classes_train=5;encoder_classifier_link=prototypical;number_of_steps=20000*',\n",
    "    '*num_classes_train=20;encoder_classifier_link=prototypical;number_of_steps=20000*',\n",
    "]\n",
    "\n",
    "# Baseline CIFAR100, simple conv net, prototypical vs cosine\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180326_170423_mini_imagenet_encoder_classifier_link_number_of_steps_metric_multiplier_init_metric_multiplier_trainable_repeat_num_classes_train_simple_conv_net_cifar100/'\n",
    "experiement_patterns = [\n",
    "    '*metric_multiplier_trainable=False*num_classes_train=5;encoder_classifier_link=cosine;number_of_steps=20000;metric_multiplier_init=1',\n",
    "    '*metric_multiplier_trainable=True*num_classes_train=5;encoder_classifier_link=cosine;number_of_steps=20000;metric_multiplier_init=1',\n",
    "    '*metric_multiplier_trainable=False*num_classes_train=5;encoder_classifier_link=cosine;number_of_steps=20000;metric_multiplier_init=15',\n",
    "    '*metric_multiplier_trainable=True*num_classes_train=5;encoder_classifier_link=cosine;number_of_steps=20000;metric_multiplier_init=15',\n",
    "    '*metric_multiplier_trainable=False*num_classes_train=20;encoder_classifier_link=cosine;number_of_steps=20000;metric_multiplier_init=1',\n",
    "    '*metric_multiplier_trainable=True*num_classes_train=20;encoder_classifier_link=cosine;number_of_steps=20000;metric_multiplier_init=1',\n",
    "    '*metric_multiplier_trainable=False*num_classes_train=20;encoder_classifier_link=cosine;number_of_steps=20000;metric_multiplier_init=15',\n",
    "    '*metric_multiplier_trainable=True*num_classes_train=20;encoder_classifier_link=cosine;number_of_steps=20000;metric_multiplier_init=15',\n",
    "    '*num_classes_train=5;encoder_classifier_link=prototypical;number_of_steps=40000*',\n",
    "    '*num_classes_train=20;encoder_classifier_link=prototypical;number_of_steps=40000*',\n",
    "    '*num_classes_train=5;encoder_classifier_link=prototypical;number_of_steps=20000*',\n",
    "    '*num_classes_train=20;encoder_classifier_link=prototypical;number_of_steps=20000*',\n",
    "]\n",
    "\n",
    "# Experiemnt with resnet feature extractor, mini-Imagenet, shot in train mode different than in the test mode\n",
    "# New baseline for the polynomial head\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180327_005425_mini_imagenet_encoder_classifier_link_number_of_steps_num_shots_train_repeat_num_classes_train_polynomial_resnet_num_shots_train_steps100000/'\n",
    "experiement_patterns = [\n",
    "    '*num_shots_train=5*num_classes_train=5*',\n",
    "    '*num_shots_train=10*num_classes_train=5*',\n",
    "]\n",
    "# Same but with 50000 steps: /mnt/home/boris/experiments_task_encoder/180326_193006_mini_imagenet_encoder_classifier_link_number_of_steps_num_shots_train_repeat_num_classes_train_polynomial_resnet_num_shots_train/\n",
    "\n",
    "# CBN first successful experiment\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180329_040057_mini_imagenet_number_of_steps_feat_extract_pretrain_feat_extract_pretrain_lr_decay_n_repeat_weight_decay_cbn_cbn_test_MoreMore/'\n",
    "experiement_patterns = [\n",
    "    '*weight_decay_cbn=0.01*number_of_steps=3*feat_extract_pretrain_lr_decay_n=2*',\n",
    "    '*weight_decay_cbn=0.01*number_of_steps=3*feat_extract_pretrain_lr_decay_n=3*',\n",
    "    '*weight_decay_cbn=0;*number_of_steps=3*feat_extract_pretrain_lr_decay_n=2*',\n",
    "    '*weight_decay_cbn=0;*number_of_steps=3*feat_extract_pretrain_lr_decay_n=3*',\n",
    "    '*weight_decay_cbn=0.0005*number_of_steps=3*feat_extract_pretrain_lr_decay_n=2*',\n",
    "    '*weight_decay_cbn=0.0005*number_of_steps=3*feat_extract_pretrain_lr_decay_n=3*',\n",
    "]\n",
    "\n",
    "# CBN vs prototypical with multitask\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180329_204205_mini_imagenet_encoder_classifier_link_number_of_steps_feat_extract_pretrain_repeat_weight_decay_cbn_cbn_vs_prototypical_multitask/'\n",
    "experiement_patterns = [\n",
    "    '*encoder_classifier_link=cbn*weight_decay_cbn=0.01*number_of_steps=30000*',\n",
    "    '*encoder_classifier_link=cbn*weight_decay_cbn=0.01*number_of_steps=20000*',\n",
    "    '*encoder_classifier_link=cbn*weight_decay_cbn=0.0005*number_of_steps=30000*',\n",
    "    '*encoder_classifier_link=cbn*weight_decay_cbn=0.0005*number_of_steps=20000*',\n",
    "    '*encoder_classifier_link=prototypical*number_of_steps=30000*',\n",
    "    '*encoder_classifier_link=prototypical*number_of_steps=20000*',\n",
    "]\n",
    "\n",
    "# CBN with polynomial head vs polynomial with multitask\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180330_022135_mini_imagenet_encoder_classifier_link_feat_extract_pretrain_metric_multiplier_trainable_repeat_weight_decay_cbn_cbn_vs_polynomial_multitask/'\n",
    "experiement_patterns = [\n",
    "    '*encoder_classifier_link=cbn*weight_decay_cbn=0.01*metric_multiplier_trainable=False*',\n",
    "    '*encoder_classifier_link=cbn*weight_decay_cbn=0.0005*metric_multiplier_trainable=False*',\n",
    "    '*encoder_classifier_link=polynomial*metric_multiplier_trainable=False*',\n",
    "    '*encoder_classifier_link=cbn*weight_decay_cbn=0.01*metric_multiplier_trainable=True*',\n",
    "    '*encoder_classifier_link=cbn*weight_decay_cbn=0.0005*metric_multiplier_trainable=True*',\n",
    "    '*encoder_classifier_link=polynomial*metric_multiplier_trainable=True*',\n",
    "]\n",
    "\n",
    "# # CBN with separate paramter encoder, with and without multitask multitask\n",
    "# ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180330_132040_mini_imagenet_encoder_classifier_link_number_of_steps_feat_extract_pretrain_repeat_weight_decay_cbn_feat_extract_pretrain_lr_decay_rate_separate_task_encoder_cbn/'\n",
    "# experiement_patterns = [\n",
    "#     '*feat_extract_pretrain=multitask;feat_extract_pretrain_lr_decay_rate=10;encoder_classifier_link=cbn;weight_decay_cbn=0.01;number_of_steps=30000*',\n",
    "#     '*feat_extract_pretrain=multitask;feat_extract_pretrain_lr_decay_rate=100;encoder_classifier_link=cbn;weight_decay_cbn=0.01;number_of_steps=30000*',\n",
    "#     '*feat_extract_pretrain=multitask;feat_extract_pretrain_lr_decay_rate=5;encoder_classifier_link=cbn;weight_decay_cbn=0.01;number_of_steps=30000*',\n",
    "#     '*feat_extract_pretrain=multitask;feat_extract_pretrain_lr_decay_rate=10;encoder_classifier_link=cbn;weight_decay_cbn=0.1;number_of_steps=30000*',\n",
    "#     '*feat_extract_pretrain=multitask;feat_extract_pretrain_lr_decay_rate=100;encoder_classifier_link=cbn;weight_decay_cbn=0.1;number_of_steps=30000*',\n",
    "#     '*feat_extract_pretrain=multitask;feat_extract_pretrain_lr_decay_rate=5;encoder_classifier_link=cbn;weight_decay_cbn=0.1;number_of_steps=30000*',\n",
    "# ]\n",
    "\n",
    "# # CBN with shared parameter encoder, optimizing the multitask parameters\n",
    "# ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180331_112834_mini_imagenet_encoder_classifier_link_number_of_steps_polynomial_metric_order_repeat_feat_extract_pretrain_decay_rate_weight_decay_cbn_feat_extract_pretrain_lr_decay_rate_shared_task_encoder_cbn_decoupling_layer/'\n",
    "# experiement_patterns = [\n",
    "#     '*pretrain_lr_decay_rate=10;*weight_decay_cbn=0.01*pretrain_decay_rate=0.9',\n",
    "#     '*pretrain_lr_decay_rate=10;*weight_decay_cbn=0.01*pretrain_decay_rate=0.95',\n",
    "# ]\n",
    "\n",
    "# # CBN with shared parameter encoder, optimizing the multitask parameters, synchronized lr annealing schedule\n",
    "# ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180331_174053_mini_imagenet_encoder_classifier_link_number_of_steps_polynomial_metric_order_repeat_feat_extract_pretrain_decay_rate_weight_decay_cbn_multitask_syncronized_lr_decay/'\n",
    "# experiement_patterns = [\n",
    "#     '*weight_decay_cbn=0.002;number_of_steps=30000;feat_extract_pretrain_decay_rate=0.95',\n",
    "#     '*weight_decay_cbn=0.002;number_of_steps=30000;feat_extract_pretrain_decay_rate=0.9',\n",
    "#     '*weight_decay_cbn=0.01;number_of_steps=30000;feat_extract_pretrain_decay_rate=0.95',\n",
    "#     '*weight_decay_cbn=0.01;number_of_steps=30000;feat_extract_pretrain_decay_rate=0.9',\n",
    "# ]\n",
    "\n",
    "# CBN baseline, 20 retrials\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180401_145432_mini_imagenet_encoder_classifier_link_number_of_steps_polynomial_metric_order_repeat_weight_decay_cbn_cbn_baseline/'\n",
    "experiement_patterns = [\n",
    "    '*number_of_steps=30000*',\n",
    "    '*number_of_steps=20000*',\n",
    "    '*number_of_steps=40000*',\n",
    "]\n",
    "\n",
    "# Multilayer CBN parameter generation and more aggressive rate decay\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180403_110651_mini_imagenet_encoder_classifier_link_number_of_steps_polynomial_metric_order_cbn_num_layers_repeat_weight_decay_cbn_cbn_more_aggressive_lr_decay/'\n",
    "experiement_patterns = [\n",
    "    '*cbn_num_layers=1*number_of_steps=30000*',\n",
    "    '*cbn_num_layers=2*number_of_steps=30000*',\n",
    "    '*cbn_num_layers=3*number_of_steps=30000*',\n",
    "]\n",
    "\n",
    "# # Dataset augmentation\n",
    "# ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180411_025808_mini_imagenet_encoder_classifier_link_feat_extract_pretrain_polynomial_metric_order_num_units_in_block_augment_number_of_steps_cbn_num_layers_repeat_augmentation_test_larger_interval/'\n",
    "# experiement_patterns = [\n",
    "#     '*augment=True;polynomial_metric_order=1*pretrain=multitask;encoder_classifier_link=cbn;number_of_steps=60000*',\n",
    "#     '*augment=True;polynomial_metric_order=1*pretrain=None;encoder_classifier_link=cbn;number_of_steps=60000*',\n",
    "#     '*augment=True;polynomial_metric_order=1*pretrain=multitask;encoder_classifier_link=polynomial;number_of_steps=60000*',\n",
    "#     '*augment=True;polynomial_metric_order=1*pretrain=None;encoder_classifier_link=polynomial;number_of_steps=60000*',\n",
    "#     '*augment=True;polynomial_metric_order=1*pretrain=multitask;encoder_classifier_link=prototypical;number_of_steps=60000*',\n",
    "#     '*augment=True;polynomial_metric_order=1*pretrain=None;encoder_classifier_link=prototypical;number_of_steps=60000*',\n",
    "    \n",
    "#     '*augment=False;polynomial_metric_order=1*pretrain=multitask;encoder_classifier_link=cbn;number_of_steps=30000*',\n",
    "#     '*augment=False;polynomial_metric_order=1*pretrain=None;encoder_classifier_link=cbn;number_of_steps=30000*',\n",
    "#     '*augment=False;polynomial_metric_order=1*pretrain=multitask;encoder_classifier_link=polynomial;number_of_steps=30000*',\n",
    "#     '*augment=False;polynomial_metric_order=1*pretrain=None;encoder_classifier_link=polynomial;number_of_steps=30000*',\n",
    "#     '*augment=False;polynomial_metric_order=1*pretrain=multitask;encoder_classifier_link=prototypical;number_of_steps=30000*',\n",
    "#     '*augment=False;polynomial_metric_order=1*pretrain=None;encoder_classifier_link=prototypical;number_of_steps=30000*',\n",
    "# ]\n",
    "\n",
    "# # CBN num layers and number of CBN per block, MIN\n",
    "# ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180412_150135_mini_imagenet_encoder_classifier_link_polynomial_metric_order_feature_extractor_number_of_steps_cbn_per_block_cbn_num_layers_repeat_residense_net_test'\n",
    "# experiement_patterns = [\n",
    "#     '*cbn_num_layers=1*cbn_per_block=False;feature_extractor=simple_res_net*',\n",
    "#     '*cbn_num_layers=1*cbn_per_block=True;feature_extractor=simple_res_net*',\n",
    "#     '*cbn_num_layers=2*cbn_per_block=False;feature_extractor=simple_res_net*',\n",
    "#     '*cbn_num_layers=2*cbn_per_block=True;feature_extractor=simple_res_net*',\n",
    "# ]\n",
    "\n",
    "# # CBN num layers and polynomial metric order, MIN\n",
    "# ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180413_023455_mini_imagenet_metric_multiplier_trainable_encoder_classifier_link_polynomial_metric_order_number_of_steps_cbn_per_block_cbn_num_layers_repeat_cbn_layers_poly_order'\n",
    "# experiement_patterns = [\n",
    "#     '*cbn_num_layers=1*polynomial_metric_order=1*cbn_per_block=True*',\n",
    "#     '*cbn_num_layers=1*polynomial_metric_order=2*cbn_per_block=True*',\n",
    "#     '*cbn_num_layers=1*polynomial_metric_order=3*cbn_per_block=True*',\n",
    "#     '*cbn_num_layers=2*polynomial_metric_order=1*cbn_per_block=True*',\n",
    "#     '*cbn_num_layers=2*polynomial_metric_order=2*cbn_per_block=True*',\n",
    "#     '*cbn_num_layers=2*polynomial_metric_order=3*cbn_per_block=True*',\n",
    "    \n",
    "#     '*cbn_num_layers=1*polynomial_metric_order=1*cbn_per_block=False*',\n",
    "#     '*cbn_num_layers=1*polynomial_metric_order=2*cbn_per_block=False*',\n",
    "#     '*cbn_num_layers=1*polynomial_metric_order=3*cbn_per_block=False*',\n",
    "#     '*cbn_num_layers=2*polynomial_metric_order=1*cbn_per_block=False*',\n",
    "#     '*cbn_num_layers=2*polynomial_metric_order=2*cbn_per_block=False*',\n",
    "#     '*cbn_num_layers=2*polynomial_metric_order=3*cbn_per_block=False*',\n",
    "# ]\n",
    "\n",
    "# # CBN num layer and one CBN per network vs one CBN per max pool\n",
    "# ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180420_022402_mini_imagenet_metric_multiplier_trainable_encoder_classifier_link_feat_extract_pretrain_polynomial_metric_order_metric_multiplier_init_cbn_per_network_cbn_per_block_cbn_num_layers_repeat_cbn_per_network_test_MIN'\n",
    "# experiement_patterns = [\n",
    "#     '*cbn_num_layers=1*polynomial_metric_order=1;cbn_per_network=True*',\n",
    "#     '*cbn_num_layers=1*polynomial_metric_order=2;cbn_per_network=True*',\n",
    "#     '*cbn_num_layers=1*polynomial_metric_order=3;cbn_per_network=True*',\n",
    "#     '*cbn_num_layers=2*polynomial_metric_order=1;cbn_per_network=True*',\n",
    "#     '*cbn_num_layers=2*polynomial_metric_order=2;cbn_per_network=True*',\n",
    "#     '*cbn_num_layers=2*polynomial_metric_order=3;cbn_per_network=True*',\n",
    "#     '*cbn_num_layers=3*polynomial_metric_order=1;cbn_per_network=True*',\n",
    "#     '*cbn_num_layers=3*polynomial_metric_order=2;cbn_per_network=True*',\n",
    "#     '*cbn_num_layers=3*polynomial_metric_order=3;cbn_per_network=True*',\n",
    "    \n",
    "#     '*cbn_num_layers=1*polynomial_metric_order=1;cbn_per_network=False*',\n",
    "#     '*cbn_num_layers=1*polynomial_metric_order=2;cbn_per_network=False*',\n",
    "#     '*cbn_num_layers=1*polynomial_metric_order=3;cbn_per_network=False*',\n",
    "#     '*cbn_num_layers=2*polynomial_metric_order=1;cbn_per_network=False*',\n",
    "#     '*cbn_num_layers=2*polynomial_metric_order=2;cbn_per_network=False*',\n",
    "#     '*cbn_num_layers=2*polynomial_metric_order=3;cbn_per_network=False*',\n",
    "#     '*cbn_num_layers=3*polynomial_metric_order=1;cbn_per_network=False*',\n",
    "#     '*cbn_num_layers=3*polynomial_metric_order=2;cbn_per_network=False*',\n",
    "#     '*cbn_num_layers=3*polynomial_metric_order=3;cbn_per_network=False*',\n",
    "# ]\n",
    "\n",
    "# 3,4,5 cbn layers, CBN per block vs CBN for every conv\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180420_191842_mini_imagenet_metric_multiplier_trainable_encoder_classifier_link_feat_extract_pretrain_polynomial_metric_order_metric_multiplier_init_cbn_per_block_cbn_num_layers_repeat_more_cbn_layers'\n",
    "experiement_patterns = [\n",
    "    '*cbn_num_layers=1*polynomial_metric_order=3*feat_extract_pretrain=multitask*cbn_per_block=False*',\n",
    "    '*cbn_num_layers=2*polynomial_metric_order=3*feat_extract_pretrain=multitask*cbn_per_block=False*',\n",
    "    '*cbn_num_layers=3*polynomial_metric_order=3*feat_extract_pretrain=multitask*cbn_per_block=False*',\n",
    "    '*cbn_num_layers=4*polynomial_metric_order=3*feat_extract_pretrain=multitask*cbn_per_block=False*',\n",
    "    '*cbn_num_layers=5*polynomial_metric_order=3*feat_extract_pretrain=multitask*cbn_per_block=False*',\n",
    "    \n",
    "    '*cbn_num_layers=1*polynomial_metric_order=3*feat_extract_pretrain=multitask*cbn_per_block=True*',\n",
    "    '*cbn_num_layers=2*polynomial_metric_order=3*feat_extract_pretrain=multitask*cbn_per_block=True*',\n",
    "    '*cbn_num_layers=3*polynomial_metric_order=3*feat_extract_pretrain=multitask*cbn_per_block=True*',\n",
    "    '*cbn_num_layers=4*polynomial_metric_order=3*feat_extract_pretrain=multitask*cbn_per_block=True*',\n",
    "    '*cbn_num_layers=5*polynomial_metric_order=3*feat_extract_pretrain=multitask*cbn_per_block=True*',\n",
    "]\n",
    "\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180502_103022_mini_imagenet_encoder_classifier_link_feat_extract_pretrain_metric_multiplier_init_feat_extract_pretrain_decay_rate_cbn_num_layers_repeat_scale_10_prev_version_test'\n",
    "experiement_patterns = [\n",
    "    '*feat_extract_pretrain_decay_rate=0.9;*',\n",
    "    '*feat_extract_pretrain_decay_rate=0.95*',\n",
    "]\n",
    "\n",
    "\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180508_015505_mini_imagenet_num_tasks_per_batch_num_max_pools_encoder_classifier_link_feat_extract_pretrain_metric_multiplier_init_cbn_num_layers_repeat_maxpool'\n",
    "experiement_patterns = [\n",
    "    '*num_tasks_per_batch=2*max_pools=3*metric_multiplier_init=10*',\n",
    "    '*num_tasks_per_batch=2*max_pools=3*metric_multiplier_init=1*',\n",
    "]\n",
    "\n",
    "# Prototypical networks 5-shot and 10-shot with adam\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180512_180858_mini_imagenet_encoder_classifier_link_number_of_steps_metric_multiplier_trainable_repeat_num_classes_train_prototypical_10shot'\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180512_190032_mini_imagenet_encoder_classifier_link_number_of_steps_metric_multiplier_trainable_repeat_num_classes_train_prototypical_5shot'\n",
    "experiement_patterns = [\n",
    "    '*number_of_steps=20000*num_classes_train=5*',\n",
    "    '*number_of_steps=20000*num_classes_train=10*',\n",
    "    '*number_of_steps=20000*num_classes_train=15*',\n",
    "    '*number_of_steps=20000*num_classes_train=20*',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf; print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get directories for the  experiements "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_experiment_folder_list(experiement_dir, experiment_pattern):\n",
    "    list_experiemnts_pattern = []\n",
    "    list_experiments_all = os.listdir(experiement_dir)\n",
    "    for folder in list_experiments_all:\n",
    "        names = os.path.join(ROOT_DIR,folder)\n",
    "#         print(names)\n",
    "        names_filt_alone = fnmatch.filter([names], experiment_pattern)\n",
    "#         print(names_filt_alone)\n",
    "        list_experiemnts_pattern.extend(names_filt_alone)\n",
    "    return list_experiemnts_pattern\n",
    "\n",
    "\n",
    "def interpolate_data(experiment_data, index_max, step):\n",
    "    index_max = int(round(index_max/step)*step)\n",
    "    index_new = np.arange(0, index_max+step, step)\n",
    "    experiment_data_interpolated = {}\n",
    "    for file, df in experiment_data.items():\n",
    "        interpolator = interpolate.interp1d(df.index, df.value, bounds_error=False, fill_value=np.nan)\n",
    "        value_new = interpolator(index_new)\n",
    "        experiment_data_interpolated[file] = value_new\n",
    "    \n",
    "    return pd.DataFrame(index=index_new, data=experiment_data_interpolated)\n",
    "    \n",
    "\n",
    "def read_experiment_data(experiement_dir, experiment_pattern):\n",
    "    experiemnt_folder_list = get_experiment_folder_list(experiement_dir, experiment_pattern)\n",
    "    experiment_data_tst = {}\n",
    "    experiment_data_val = {}\n",
    "    index_max_tst = 0\n",
    "    index_max_val = 0\n",
    "    \n",
    "    i=0    \n",
    "    for experiement in experiemnt_folder_list:\n",
    "        print(experiement)\n",
    "        print(glob.glob1(experiement+'/eval/', 'events.out.tfevents.*'))\n",
    "        df_tst=[]\n",
    "        df_val=[]\n",
    "        for file in glob.glob1(experiement+'/eval/', 'events.out.tfevents.*'):\n",
    "            ea = EventAccumulator(experiement+'/eval/'+file)\n",
    "            ea.Reload()\n",
    "            tags = ea.Tags()['scalars']\n",
    "#             print(tags)\n",
    "            if not 'accuracy_target_tst' in tags:\n",
    "                continue\n",
    "            data = ea.Scalars('accuracy_target_tst')\n",
    "            if len(data) > 0:\n",
    "                df_tst.append(pd.DataFrame(data).set_index('step').drop('wall_time', axis=1))\n",
    "            data = ea.Scalars('accuracy_target_val')\n",
    "            if len(data) > 0:\n",
    "                df_val.append(pd.DataFrame(data).set_index('step').drop('wall_time', axis=1))\n",
    "        \n",
    "        if len(df_tst) > 0:\n",
    "            df_tst = pd.concat(df_tst)\n",
    "\n",
    "        if len(df_val) > 0:\n",
    "            df_val = pd.concat(df_val)\n",
    "        \n",
    "        if len(df_tst) > 0:\n",
    "            experiment_data_tst[experiement+file] = df_tst\n",
    "            index_max_tst = max(index_max_tst, df_tst.index.max())\n",
    "        \n",
    "        if len(df_tst) > 0:\n",
    "            experiment_data_val[experiement+file] = df_val\n",
    "            index_max_val = max(index_max_val, df_val.index.max())\n",
    "        \n",
    "    return interpolate_data(experiment_data_tst, index_max_tst, step=250), interpolate_data(experiment_data_val, index_max_val, step=250)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_experiemnts_target_transfer = get_experiment_folder_list(ROOT_DIR, experiement_patterns[0])\n",
    "list_experiemnts_target_transfer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_data = {}\n",
    "for experiment_pattern in experiement_patterns:\n",
    "    print('Loading experiment: %s' %experiment_pattern)\n",
    "    acc_tst, acc_val = read_experiment_data(ROOT_DIR, experiment_pattern)\n",
    "    \n",
    "    plot_data[(experiment_pattern, 'acc_tst')] = acc_tst\n",
    "    plot_data[(experiment_pattern, 'acc_val')] = acc_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is to detect runs that did not converge\n",
    "VALID_RUN_THRESHOLD=0.0\n",
    "\n",
    "for experiment_pattern in experiement_patterns:\n",
    "    print('Loading experiment: %s' %experiment_pattern)\n",
    "    \n",
    "    acc_tst = plot_data[(experiment_pattern, 'acc_tst')]\n",
    "    acc_val = plot_data[(experiment_pattern, 'acc_val')]\n",
    "    \n",
    "    valid_cols = acc_val.max().index[acc_val.max() > VALID_RUN_THRESHOLD]\n",
    "    acc_val = acc_val[valid_cols]\n",
    "    acc_tst = acc_tst[valid_cols]\n",
    "    \n",
    "    acc_tst_ci = sms.DescrStatsW(acc_tst.T).tconfint_mean()\n",
    "    acc_val_ci = sms.DescrStatsW(acc_val.T).tconfint_mean()\n",
    "    acc_val_mean = acc_val.mean(axis=1)\n",
    "    acc_tst_mean = acc_tst.mean(axis=1)\n",
    "    \n",
    "    plot_data[(experiment_pattern, 'acc_tst_mean')] = acc_tst_mean\n",
    "    plot_data[(experiment_pattern, 'acc_val_mean')] = acc_val_mean\n",
    "    plot_data[(experiment_pattern, 'acc_tst_ci')] = acc_tst_ci\n",
    "    plot_data[(experiment_pattern, 'acc_val_ci')] = acc_val_ci\n",
    "    plot_data[(experiment_pattern, 'acc_tst')] = acc_tst\n",
    "    plot_data[(experiment_pattern, 'acc_val')] = acc_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "colors=[colorsys.hsv_to_rgb(x*1.0/len(experiement_patterns), 1.0, 1.0) for x in range(len(experiement_patterns))]\n",
    "\n",
    "majorLocator = MultipleLocator(1.0)\n",
    "majorFormatter = FormatStrFormatter('%.1f')\n",
    "\n",
    "\n",
    "fig = plt.figure(figsize=(20,20))\n",
    "ax1 = fig.add_subplot(2,1,1)\n",
    "ax2 = fig.add_subplot(2,1,2)\n",
    "for i, experiment_pattern in enumerate(experiement_patterns):\n",
    "    \n",
    "\n",
    "    ax1.plot(100.0*plot_data[(experiment_pattern, 'acc_tst_mean')], label=experiment_pattern, c=colors[i])\n",
    "    ax1.fill_between(plot_data[(experiment_pattern, 'acc_tst_mean')].index, \n",
    "                    100.0*plot_data[(experiment_pattern, 'acc_tst_ci')][0], \n",
    "                    100.0*plot_data[(experiment_pattern, 'acc_tst_ci')][1], \n",
    "                    alpha=0.5, edgecolor=colors[i], facecolor=colors[i])\n",
    "\n",
    "    legend = ax1.legend(shadow=True, loc=4)\n",
    "    ax1.yaxis.set_major_formatter(FormatStrFormatter('%.2f'))\n",
    "\n",
    "    ax1.grid(which='minor', alpha=0.5)                                                \n",
    "    ax1.grid(which='major', alpha=1.0) \n",
    "    ax1.set_ylim([65,80])\n",
    "\n",
    "    ax1.set_title(ROOT_DIR)\n",
    "    ax1.set_xlabel('train iterations')\n",
    "    ax1.set_xticks(range(0, 50001, 5000))\n",
    "    ax1.set_ylabel('Accuracy target test')\n",
    "    ax1.yaxis.set_major_locator(majorLocator)\n",
    "    ax1.yaxis.set_major_formatter(majorFormatter)\n",
    "    plt.xticks(rotation=45)\n",
    "    \n",
    "    \n",
    "\n",
    "for i, experiment_pattern in enumerate(experiement_patterns):\n",
    "    \n",
    "\n",
    "    ax2.plot(100.0*plot_data[(experiment_pattern, 'acc_val_mean')], label=experiment_pattern, c=colors[i])\n",
    "    ax2.fill_between(plot_data[(experiment_pattern, 'acc_val_mean')].index, \n",
    "                    100.0*plot_data[(experiment_pattern, 'acc_val_ci')][0], \n",
    "                    100.0*plot_data[(experiment_pattern, 'acc_val_ci')][1], \n",
    "                    alpha=0.5, edgecolor=colors[i], facecolor=colors[i])\n",
    "\n",
    "    legend = ax2.legend(shadow=True, loc=4)\n",
    "    ax2.yaxis.set_major_formatter(FormatStrFormatter('%.2f'))\n",
    "\n",
    "    ax2.grid(which='minor', alpha=0.5)                                                \n",
    "    ax2.grid(which='major', alpha=1.0) \n",
    "    ax2.set_ylim([65,80])\n",
    "\n",
    "    ax2.set_title(ROOT_DIR)\n",
    "    ax2.set_xlabel('train iterations')\n",
    "    ax2.set_xticks(range(0, 50001, 5000))\n",
    "    ax2.set_ylabel('Accuracy target validation')\n",
    "    ax2.yaxis.set_major_locator(majorLocator)\n",
    "    ax2.yaxis.set_major_formatter(majorFormatter)\n",
    "    plt.xticks(rotation=45)\n",
    "        \n",
    "\n",
    "plt.grid('on')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "acc_tst.loc[acc_tst.idxmax(axis=1).dropna().values==acc_tst.columns[0], acc_tst.columns[0]]\n",
    "\n",
    "\n",
    "def get_best_over_val(acc_tst, acc_val):\n",
    "    df = pd.DataFrame(index=acc_tst.index, columns=['acc_tst'])\n",
    "    for column in acc_val.columns:\n",
    "        idx=acc_val.idxmax(axis=1).dropna().values==column\n",
    "        df.loc[idx,'acc_tst'] = acc_tst.loc[idx, column].values\n",
    "    return df\n",
    "\n",
    "\n",
    "fig = plt.figure(figsize=(20,20))\n",
    "ax1 = fig.add_subplot(2,1,1)\n",
    "ax1.set_ylim([60,75])\n",
    "plt.grid('on')\n",
    "for i, experiment_pattern in enumerate(experiement_patterns):\n",
    "    acc_tst, acc_val = plot_data[(experiment_pattern, 'acc_tst')], plot_data[(experiment_pattern, 'acc_val')]\n",
    "    acc_tst_best = get_best_over_val(acc_tst, acc_val)\n",
    "    ax1.plot(100*acc_tst_best, label=experiment_pattern, c=colors[i])\n",
    "    legend = ax1.legend(shadow=True, loc=4)\n",
    "    \n",
    "    ax1.set_title(ROOT_DIR)\n",
    "    ax1.set_xlabel('train iterations')\n",
    "    ax1.set_xticks(range(0, 50001, 10000))\n",
    "    ax1.set_ylabel('Accuracy target test, best over validation set')\n",
    "    ax1.yaxis.set_major_locator(majorLocator)\n",
    "    ax1.yaxis.set_major_formatter(majorFormatter)\n",
    "    plt.xticks(rotation=45)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def argmax_to_max(arr, argmax, axis):\n",
    "    \"\"\"argmax_to_max(arr, arr.argmax(axis), axis) == arr.max(axis)\"\"\"\n",
    "    new_shape = list(arr.shape)\n",
    "    del new_shape[axis]\n",
    "\n",
    "    grid = np.ogrid[tuple(map(slice, new_shape))]\n",
    "    grid.insert(axis, argmax)\n",
    "\n",
    "    return arr[tuple(grid)]\n",
    "\n",
    "\n",
    "def get_max_over_val_bootstrapped(acc_tst, acc_val, bootstrap_ratio, n_mc_trials, replace):\n",
    "    acc_tst = acc_tst.values\n",
    "    acc_val = acc_val.values\n",
    "    assert acc_val.shape[1] == acc_tst.shape[1]\n",
    "    output = np.zeros(shape=(acc_tst.shape[0], n_mc_trials))\n",
    "    for i in range(n_mc_trials):\n",
    "        col_dxs = np.random.choice(range(acc_val.shape[1]), size=round(bootstrap_ratio*acc_val.shape[1]), \n",
    "                                  replace=replace)\n",
    "        acc_val_boot = acc_val[:,col_dxs]\n",
    "        acc_tst_boot = acc_tst[:,col_dxs]\n",
    "                \n",
    "        val_max_idx = np.argmax(acc_val_boot, axis=1)\n",
    "        output[:, i] = argmax_to_max(acc_tst_boot, val_max_idx, axis=1)\n",
    "\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(20,20))\n",
    "ax1 = fig.add_subplot(2,1,1)\n",
    "ax1.set_ylim([60,75])\n",
    "ax2 = fig.add_subplot(2,1,2)\n",
    "ax2.set_ylim([65,80])\n",
    "plt.grid('on')\n",
    "for i, experiment_pattern in enumerate(experiement_patterns):\n",
    "    acc_tst, acc_val = plot_data[(experiment_pattern, 'acc_tst')], plot_data[(experiment_pattern, 'acc_val')]\n",
    "    a = get_max_over_val_bootstrapped(acc_tst, acc_val, bootstrap_ratio=1.0, \n",
    "                                      n_mc_trials=1000, replace=True\n",
    "                                     )\n",
    "    \n",
    "    a_ci = np.percentile(a, [5.0, 95.0], axis=1)\n",
    "    \n",
    "    ax1.plot(acc_tst.index, 100*np.mean(a, axis=1), label=experiment_pattern, c=colors[i])\n",
    "    \n",
    "    ax1.fill_between(acc_tst.index, 100.0*a_ci[0], 100.0*a_ci[1], \n",
    "                    alpha=0.5, edgecolor=colors[i], facecolor=colors[i])\n",
    "    \n",
    "    legend = ax1.legend(shadow=True, loc=4)\n",
    "    ax1.grid(which='minor', alpha=0.5)                                                \n",
    "    ax1.grid(which='major', alpha=1.0) \n",
    "    ax1.set_ylim([65,80])\n",
    "    \n",
    "    ax1.set_title(ROOT_DIR)\n",
    "    ax1.set_xlabel('train iterations')\n",
    "    ax1.set_xticks(range(0, 50001, 5000))\n",
    "    ax1.set_ylabel('Accuracy target test, best over validation set')\n",
    "    ax1.yaxis.set_major_locator(majorLocator)\n",
    "    ax1.yaxis.set_major_formatter(majorFormatter)\n",
    "    plt.xticks(rotation=45)\n",
    "    \n",
    "    \n",
    "for i, experiment_pattern in enumerate(experiement_patterns):\n",
    "    acc_tst, acc_val = plot_data[(experiment_pattern, 'acc_val')], plot_data[(experiment_pattern, 'acc_val')]\n",
    "    a = get_max_over_val_bootstrapped(acc_tst, acc_val, bootstrap_ratio=1.0, \n",
    "                                      n_mc_trials=10000, replace=True\n",
    "                                     )\n",
    "    \n",
    "    a_ci = np.percentile(a, [5.0, 95.0], axis=1)\n",
    "    \n",
    "    ax2.plot(acc_tst.index, 100*np.mean(a, axis=1), label=experiment_pattern, c=colors[i])\n",
    "    \n",
    "    ax2.fill_between(acc_tst.index, 100.0*a_ci[0], 100.0*a_ci[1], \n",
    "                    alpha=0.5, edgecolor=colors[i], facecolor=colors[i])\n",
    "    \n",
    "    legend = ax2.legend(shadow=True, loc=4)\n",
    "    ax2.grid(which='minor', alpha=0.5)                                                \n",
    "    ax2.grid(which='major', alpha=1.0) \n",
    "    ax2.set_ylim([65,80])\n",
    "    \n",
    "    ax2.set_title(ROOT_DIR)\n",
    "    ax2.set_xlabel('train iterations')\n",
    "    ax2.set_xticks(range(0, 50001, 5000))\n",
    "    ax2.set_ylabel('Accuracy target validation, best over validation set')\n",
    "    ax2.yaxis.set_major_locator(majorLocator)\n",
    "    ax2.yaxis.set_major_formatter(majorFormatter)\n",
    "    plt.xticks(rotation=45)\n",
    "    \n",
    "plt.grid('on')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def get_early_stop_over_val(acc_tst, acc_val):\n",
    "    acc_tst = np.nan_to_num(acc_tst.values)\n",
    "    acc_val = np.nan_to_num(acc_val.values)\n",
    "    assert acc_val.shape[1] == acc_tst.shape[1]\n",
    "        \n",
    "    val_max_idx = np.argmax(acc_val, axis=0)\n",
    "    acc_tst_max = argmax_to_max(acc_tst, val_max_idx, axis=0)\n",
    "    \n",
    "    return acc_tst_max\n",
    "\n",
    "print(\"Experiment root folder: %s\" %ROOT_DIR)\n",
    "for i, experiment_pattern in enumerate(experiement_patterns):\n",
    "    acc_tst, acc_val = plot_data[(experiment_pattern, 'acc_tst')], plot_data[(experiment_pattern, 'acc_val')]\n",
    "    acc_tst_validated = get_early_stop_over_val(acc_tst, acc_val)\n",
    "    \n",
    "    acc_tst_validated_ci = sms.DescrStatsW(acc_tst_validated.T).tconfint_mean()\n",
    "    acc_tst_validated_mean = acc_tst_validated.mean()\n",
    "    \n",
    "    print('Experiment name: %s' %(experiment_pattern))\n",
    "    print('mean accuracy: %.1f +/- %.1f' %(100.0*acc_tst_validated_mean, \n",
    "            100.0*(acc_tst_validated_ci[1]-acc_tst_validated_mean)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Experiment root folder: %s\" %ROOT_DIR)\n",
    "for i, experiment_pattern in enumerate(experiement_patterns):\n",
    "    acc_tst, acc_val = plot_data[(experiment_pattern, 'acc_tst')], plot_data[(experiment_pattern, 'acc_val')]\n",
    "    acc_tst_validated = get_early_stop_over_val(acc_val, acc_val)\n",
    "    \n",
    "    acc_tst_validated_ci = sms.DescrStatsW(acc_tst_validated.T).tconfint_mean()\n",
    "    acc_tst_validated_mean = acc_tst_validated.mean()\n",
    "    \n",
    "    print('Experiment name: %s' %(experiment_pattern))\n",
    "    print('mean accuracy, validation: %.1f +/- %.1f' %(100.0*acc_tst_validated_mean, \n",
    "            100.0*(acc_tst_validated_ci[1]-acc_tst_validated_mean)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    ""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    ""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    ""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}